# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import pymysql
import pymongo
import scrapy
import re
from scrapy.pipelines.images import ImagesPipeline
from scrapy.exceptions import DropItem
import json


class CsdnPipeline(object):
    def process_item(self, item, spider):
        """必须包含的项目管道"""
        if item['number'] is None:  # 去除观看人数为空的课程，人数为空表示该课程或为预约课
            raise DropItem("Drop item found: %s" % item)
        else:
            return item


class JsonWriterPipeline(object):
    """保存为.jl文件，详情见 https://docs.scrapy.org/en/latest/topics/item-pipeline.html#write-items-to-a-json-file"""

    def __init__(self):
        self.file = open('./course.jl', 'w', encoding='utf-8')  # 打开sina.jl文件

    def open_spider(self, spider):  # 方法名均为固定写法写法
        pass

    def close_spider(self, spider):  # 关闭sina.jl文件
        self.file.close()

    def process_item(self, item, spider):
        line = json.dumps(dict(item), ensure_ascii=False) + "\n"  # 将item转换为字典，然后序列化为json格式的字符串
        self.file.write(line)
        return item


class MongoPipeline(object):
    """保存到MongoDB，详情见 https://docs.scrapy.org/en/latest/topics/item-pipeline.html#write-items-to-mongodb"""
    collection_name = 'course'  # 集合名（"表"）

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db

    @classmethod
    def from_crawler(cls, crawler):
        """信息注入，包括设置MongoDB地址与库名"""
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )

    def open_spider(self, spider):
        """打开MongoDB数据库"""
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]

    def process_item(self, item, spider):
        """插入数据"""
        # 保存报错must not contain '.',将key的核对关闭，默认打开
        self.db[self.collection_name].insert(dict(item), check_keys=False)
        return item

    def close_spider(self, spider):
        """关闭MongoDB数据库"""
        self.client.close()


class MyImagesPipeline(ImagesPipeline):
    """图片下载，详情见 https://docs.scrapy.org/en/latest/topics/media-pipeline.html#custom-images-pipeline-example"""

    def get_media_requests(self, item, info):
        """获取要下载的图像的URL，并由管道完成请求，然后下载"""
        yield scrapy.Request(url=item['pic'])

    def file_path(self, request, response=None, info=None):
        """返回图片下载后保存的名称，没有此方法Scrapy则自动给一个唯一值作为图片名称"""
        url = request.url
        file_name = re.findall('net/(.*?)g', url)[0] + 'g'
        return file_name

    def item_completed(self, results, item, info):
        """下载请求完成后调用此方法"""
        image_paths = [x['path'] for ok, x in results if ok]
        if not image_paths:
            raise DropItem("Item contains no images")
        item['picname'] = image_paths[0]  # 返回图片名
        return item
